{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of stop word lists : 52\n",
      "number of stop words in all the lists : 2066\n"
     ]
    }
   ],
   "source": [
    "stop_word_dict = {}\n",
    "file_path = \"stopwords/en/\"\n",
    "for stop_word_file in os.listdir(file_path):\n",
    "    # _none.txt doesn't contain any stop word\n",
    "    if stop_word_file == \"_none.txt\":\n",
    "        continue\n",
    "    # these files contain ngrams, not just single words\n",
    "    if stop_word_file in [\"galago_structured.txt\", \"gilner_morales.txt\"]:\n",
    "        continue\n",
    "    # This is meant to augment another list, and so is confusing here\n",
    "    if stop_word_file == \"galago_forumstop.txt\":\n",
    "        continue\n",
    "    input_file = open(file_path + stop_word_file, encoding='UTF-8')\n",
    "    stop_words = input_file.readlines()\n",
    "    input_file.close()\n",
    "    stop_words = list(map(lambda x:x.strip().lower(), stop_words))\n",
    "    stop_words = list(filter(lambda x:len(x) > 0, stop_words))\n",
    "    stop_words = list(filter(lambda x:x[0] >= 'a' and x[0] <= 'z', stop_words))\n",
    "    # there are some duplicate stop words\n",
    "    stop_words = list(set(stop_words))\n",
    "    # calculate frequency\n",
    "    for stop_word in stop_words:\n",
    "        stop_word_dict[stop_word] = stop_word_dict.get(stop_word, 0) + 1\n",
    "print(\"number of stop word lists : %d\" %(len(os.listdir(file_path)) - 4))\n",
    "print(\"number of stop words in all the lists : %d\" %(len(stop_word_dict.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "freq_df = pd.DataFrame({\"word\":list(stop_word_dict.keys()), \"freq\":list(stop_word_dict.values())})\n",
    "freq_df[\"percentage\"] = freq_df[\"freq\"] / 52"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of stop words included in <10% lists : 1396; percentage : 67.57%\n",
      "number of stop words included in 1 list : 807; percentage : 39.06%\n",
      "number of stop words included in >80% lists : 64; percentage : 3.10%\n"
     ]
    }
   ],
   "source": [
    "controversial_word = list(freq_df[freq_df.percentage < 0.1]['word'])\n",
    "print(\"number of stop words included in <10%% lists : %d; percentage : %.2f%%\"\n",
    "      %(len(controversial_word), len(controversial_word) * 100 / freq_df.shape[0]))\n",
    "word_in_one_list = list(freq_df[freq_df.freq == 1]['word'])\n",
    "print(\"number of stop words included in 1 list : %d; percentage : %.2f%%\"\n",
    "      %(len(word_in_one_list), len(word_in_one_list) * 100 / freq_df.shape[0]))\n",
    "accpeted_word = list(freq_df[freq_df.percentage > 0.8]['word'])\n",
    "print(\"number of stop words included in >80%% lists : %d; percentage : %.2f%%\"\n",
    "      %(len(accpeted_word), len(accpeted_word) * 100 / freq_df.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of stop lists with controversial words :  45\n"
     ]
    }
   ],
   "source": [
    "stop_word_dict = {}\n",
    "file_path = \"stopwords/en/\"\n",
    "controversial_list_cnt = 0\n",
    "for stop_word_file in os.listdir(file_path):\n",
    "    # _none.txt doesn't contain any stop word\n",
    "    if stop_word_file == \"_none.txt\":\n",
    "        continue\n",
    "    # these files contain ngrams, not just single words\n",
    "    if stop_word_file in [\"galago_structured.txt\", \"gilner_morales.txt\"]:\n",
    "        continue\n",
    "    # This is meant to augment another list, and so is confusing here\n",
    "    if stop_word_file == \"galago_forumstop.txt\":\n",
    "        continue\n",
    "    input_file = open(file_path + stop_word_file, encoding='UTF-8')\n",
    "    stop_words = input_file.readlines()\n",
    "    input_file.close()\n",
    "    stop_words = list(map(lambda x:x.strip().lower(), stop_words))\n",
    "    stop_words = list(filter(lambda x:len(x) > 0, stop_words))\n",
    "    stop_words = list(filter(lambda x:x[0] >= 'a' and x[0] <= 'z', stop_words))\n",
    "    # there are some duplicate stop words\n",
    "    stop_words = list(set(stop_words))\n",
    "    # calculate frequency\n",
    "    for word in stop_words:\n",
    "        if word in controversial_word:\n",
    "            controversial_list_cnt += 1\n",
    "            break\n",
    "print(\"number of stop lists with controversial words : \", controversial_list_cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"google_1gram_cnt.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAADQCAYAAAA53LuNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFxZJREFUeJzt3XnUHFWZx/HvD8ImBMISSGRLgKDi\nyPqyKI5sygFUAg4IjAJiNI6yjssA4gyugwqIoohmAA2IQIiDBg4qEEBkCRBI2GEIECACIQImLAIG\nnvmjbpPOm367612qq5ff55w+XXX7dvXTRfJQuV33PooIzMys+ZYrOwAzs27lBGxmVhInYDOzkjgB\nm5mVxAnYzKwkTsBmZiVxAjYzK4kTsJlZSZyAzcxKMqzsAAZjnXXWiTFjxpQdhpnZUu64446/RsTI\nRv3aOgGPGTOGmTNnlh2GmdlSJD2ep5+HIMzMSuIEbGZWEidgM7OSOAGbmZXECdjMrCROwGZmJem6\nBDxq1CgkDflj1KhRZX81M2szXZeA58+f31bHNbPO1XUJ2MysVTgBm5mVxAnYzKwkTsBmZiVxAjYz\nK4kTsJlZSQpNwJJGSJoq6UFJD0h6r6S1JF0t6eH0vGbqK0lnSpoj6W5J2xYZm5lZ2Yq+Av4R8IeI\neCewFfAAcAIwPSLGAdPTPsDewLj0mAicXXBsZmalKiwBS1od+ABwLkBEvB4RfwPGA5NTt8nAfml7\nPHB+ZGYAIySNLio+M7OyFXkFvAmwAPiFpFmSzpG0KrBeRDwNkJ7XTf3XB56sev+81GZm1pGKTMDD\ngG2BsyNiG+Bllgw31KIabbFMJ2mipJmSZi5YsGBoIjUzK0GRCXgeMC8ibk37U8kS8vzK0EJ6fraq\n/4ZV798AeKr3QSNiUkT0RETPyJENa96ZmbWswhJwRDwDPCnpHalpD+B+YBpweGo7HPhd2p4GHJbu\nhtgJWFgZqjAz60RFV0U+GrhQ0orAo8ARZEl/iqQJwBPAganvlcA+wBzgldTXzKxjFZqAI2I20FPj\npT1q9A3gyCLjMTNrJQ2HICRtKmmltL2rpGMkjSg+NDOzzpZnDPg3wBuSNiO7p3cs8OtCozIz6wJ5\nEvCbEbEY2B/4YUT8O+AJEmZmg5QnAf9D0iFkdyxckdpWKC4kM7PukCcBHwG8F/hORDwmaSzwq2LD\nMjPrfA3vgoiI+4FjqvYfA75bZFBmZt2gzwQs6R5qTAWuiIgtC4nIzKxL1LsC/kh6rtybe0F6/gTZ\nRAkzMxuEPhNwRDwOIGnniNi56qUTJN0EfLPo4MzMOlmeH+FWlfT+yo6k9wGrFheSmVl3yDMV+dNk\na/quQTYmvDC1mZnZINRNwJKWAzaLiK1ShQtFxMLmhGZm1tnqDkFExJvAUWl7kZOvmdnQyTMGfLWk\nL0vaMFU0XkvSWoVHZmbW4fKOAcPSS0UGWc03MzMboDwz4cYO9OCS5gIvAm8AiyOiJ109XwKMAeYC\nH4+IFySJrIz9PmT3GX8qIu4c6GebmbW6POsBr5DWAJ6aHkdJ6s9iPLtFxNYRUVmY/QRgekSMA6az\npFDn3sC49JgInN2PzzAzazt5xoDPBrYDfpoe2zG45DgemJy2JwP7VbWfH5kZwIhK8U4zs06UZwx4\n+4jYqmr/Wkl35Tx+AFdJCuDnETEJWK9SbDMinpa0buq7PvBk1XvnpbalCnNKmkh2hcxGG22UMwwz\ns9aTJwG/IWnTiHgEQNImZGO6eewcEU+lJHu1pAfr9FWNtmUWA0pJfBJAT09Pn4sFmZm1ujwJ+CvA\ndZIeJUuSG5OzYnFEPJWen5V0GbADMF/S6HT1Oxp4NnWfB2xY9fYNgKfyfQ0zs/bTcAw4IqaT/TB2\nTHq8IyKua/Q+SatKGl7ZBvYE7gWmkVXXID3/Lm1PAw5TZidgYWWowsysE9VbD/gu4EbgZuCmiLi7\nn8deD7gsu7uMYcCvI+IPkm4HpkiaADwBHJj6X0l2C9ocstvQcl1lm5m1q3pDEJ8A3gd8CDg5XcXe\nXHlExK31DhwRjwJb1Wh/DtijRnuw9GQPM7OOVm894HvJhgwmAUhaBzgYOA44DVi+GQGamXWqekMQ\nywPbkF0F7wxsCvwFOAe4pSnRmZl1sHpDEIuAB4CzgBNSMU4zMxsi9RLwZ8jK0X8GOCL9eHYLcEtE\n/KUZwZmZdbJ6Y8AXARcBSHob2T28OwOnSFoxIjZuTohmZp2pUUWMVYEdWTIOvD3ZdOGbig/NzKyz\n1fsRbhawEVAZejgdmBERLzUpNjOzjlbvCvhw4J50f66ZmQ2xemPA/Z35ZmZm/ZBnPWAzMyuAE7CZ\nWUkaLkcpaWXgC8D7ydbnvRE4OyJeLTg2M7OOlmc94PPJCmv+OO0fAlzAklXMzMxsAPIk4Hf0Kkl0\nXT9KEpmZWR/yjAHPSgukAyBpR/oxEUPS8pJmSboi7Y+VdKukhyVdImnF1L5S2p+TXh/Tv69iZtZe\n8iTgHYGbJc2VNJdsUsYuku6RlOdWtWPJFvWp+B5wRipL/wIwIbVPAF6IiM2AM1I/M7OOlWcIYq+B\nHlzSBsCHge8AX1RWHmN34F9Tl8nA18nK3I9P2wBTgZ9IkieCmFmnapiAI+JxSVsB/5ya/hwReceA\nfwj8BzA87a8N/C0iFqf9Sul5qCpLHxGLJS1M/f9afUCXpTezTtFwCELSscCFwLrp8StJR+d430eA\nZyPijurmGl0jx2tLGiImRURPRPSMHDmyURhmZi0rzxDEBGDHiHgZQNL3yMaBf1z3XdnqaftK2gdY\nGVid7Ip4hKRh6Sq4uvR8pSz9PEnDgDWA5/v5fczM2kaeH+EEvFG1/wa1r1aXEhEnRsQGETGGrJbc\ntRHxCeA64IDUrXdZ+kq5+gNSf4//mlnHynMF/AvgVkmXpf39gPMG8ZnHAxdL+jYwCzg3tZ8LXCBp\nDtmV78GD+Awzs5anPBeZkrYlm4os4IaImFV0YHn09PTEzJkz+/We7EaMYviC3cwAJN0RET2N+uVZ\nC+KCiDgUuLNGm5mZDVCeMeB3V++kcvXbFROOmVn36DMBSzpR0ovAlpIWSXox7T/Lkh/OzMxsgPpM\nwBFxSkQMB06NiNUjYnh6rB0RJzYxRjOzjlSvKOfGZLPWTkz7u5HdATEXOCsiXm9KhGZmHareGPAU\nYFUASVsDlwJPAFsDPy0+NDOzzlbvLohVIqIyS+2TwHkRcbqk5YDZxYdmZtbZ6l0BV98wuzswHSAi\n3iw0IjOzLlHvCvhaSVOAp4E1gWsBJI0GPP5rZjZI9RLwccBBwGjg/RHxj9Q+Cjip6MDMzDpdnwk4\nLYRzcY32lpiGbGbW7vLMhDMzswI4AZuZlaTeVOTp6dnFMc3MClDvR7jRknYhq2pxMb0WYY+IO2u/\nzczM8qiXgP8LOIGsbNAPer0WZPcG90nSysANwErpc6ZGxMmSxpL9uLcW2RKXh0bE65JWAs4nW2nt\nOeCgiJjb729kZtYm6i3GMzUi9ga+HxG79XrUTb7Ja8DuEbEV2fTlvSTtBHwPOCMixgEvkNWcIz2/\nEBGbAWekfmZmHavhj3AR8S1J+0o6LT0+kufAkXkp7a6QHpUr56mpfTLZAj8A49M+6fU9VGT5CjOz\nkuUpS38KcCxwf3ocm9oakrS8pNlkawhfDTxCtsLa4tRlHrB+2l4feBIgvb4QWLvGMSdKmilp5oIF\nC/KEYWbWkvLchvZh4EMRcV5EnAfsldoaiog3ImJrsnHkHYB31eqWnmtd7S5TZC0iJkVET0T0jBw5\nMk8YZmYtKe99wCOqttfo74dExN+A64GdgBGSKj/+bQBUVlybB2wIkF5fg6w6splZR8qTgE8BZkn6\npaTJwB3Afzd6k6SRkkak7VWADwIPANcBB6Ruh7OkvNG0tE96/dpwmWEz62ANqyJHxEWSrge2Jxsm\nOD4inslx7NHA5FTEczlgSkRcIel+4GJJ3wZmAeem/ucCF0iaQ3ble3C/v42ZWRtpmIABIuJpsivU\n3CLibmCbGu2Pko0H925/FTiwP59hZtbOvBaEmVlJnIDNzEpSNwFLWk7Svc0Kxsysm9RNwKn+212S\nNmpSPGZmXSPPj3Cjgfsk3Qa8XGmMiH0Li8rMrAvkScDfKDwKM7MulOc+4D9J2hgYFxHXSHobsHzx\noZmZdbY8i/F8lmx1sp+npvWB3xYZlJlZN8hzG9qRwM7AIoCIeBhYt8igzMy6QZ4E/FpEvF7ZSQvl\neI0GM7NBypOA/yTpq8Aqkj4EXApcXmxYZmadL08CPgFYANwDfA64EvhakUGZmXWDPHdBvJmWobyV\nbOjhIS8TaWY2eA0TsKQPAz8jKyckYKykz0XE74sOzsysk+UZgjgd2C0ido2IXYDdyKoW1yVpQ0nX\nSXpA0n2Sjk3ta0m6WtLD6XnN1C5JZ0qaI+luSdsO5ouZmbW6PAn42YiYU7X/KFmRzUYWA1+KiHeR\nlSI6UtIWZGPK01NZ+ulpH2BvYFx6TATOzvcVzMzaU59DEJI+ljbvk3QlMIVsDPhA4PZGB06LuD+d\ntl+U9ADZJI7xwK6p22SyWnHHp/bz0/jyDEkjJI1OxzEz6zj1xoA/WrU9H9glbS8A1uzPh0gaQ1Yd\n41ZgvUpSjYinJVUmdbxVlj6plKxfKgFLmkh2hcxGG3mRNjNrX30m4Ig4Yig+QNJqwG+A4yJikVSr\n+nzWtVYYNeKaBEwC6Onp8d0YZta28twFMRY4GhhT3T/PcpSSViBLvhdGxP+m5vmVoQVJo1kynvxW\nWfqkumS9mVnHybMc5W/JKhZfDryZ98DKLnXPBR6IiB9UvVQpP/9dli1Lf5Ski4EdgYUe/zWzTpYn\nAb8aEWcO4Ng7A4cC90iandq+SpZ4p0iaADzBkkrIVwL7AHOAV4AhGQIxM2tVeRLwjySdDFwFvFZp\njIg7670pIm6k9rguwB41+gfZymtmZl0hTwJ+D9mV7O4sGYKItG9mZgOUJwHvD2xSvSSlmZkNXp6Z\ncHcBI4oOxMys2+S5Al4PeFDS7Sw9BuyqyGZmg5AnAZ9ceBRmZl0oV1XkZgRiZtZt8syEe5ElU4JX\nBFYAXo6I1YsMzMys0+W5Ah5evS9pP2CHwiIyM+sSee6CWEpE/BbfA2xmNmh5hiA+VrW7HNCDy9Jb\nk40aNYr58+cP+XHXW289nnnmmSE/rlkeee6CqF4XeDEwl2zxdOtyRSTFvhJiEcm3yOOa5ZFnDNiL\n4lhNRSQvJ0TrJvVKEv1XnfdFRHyrgHjMzLpGvR/hXq7xAJhAVsPNchg1ahSShvwxatSoUj/LzAav\nzwQcEadXHmQlgFYhW6P3YmCTRgeWdJ6kZyXdW9XWdSXpmzl26XFSs/ZS9za0lDC/DdxNNlyxbUQc\nHxF5ytL/EtirV5tL0lvL878krFn6TMCSTiUrP/8i8J6I+HpEvJD3wBFxA/B8r+bxZKXoSc/7VbWf\nH5kZwIhUL86s6fwvCWuWelfAXwLeDnwNeErSovR4UdKiAX7eUiXpgUYl6ZchaaKkmZJmLliwYIBh\nmJmVr15Z+n7PkhuEXCXpwWXpzaxzNDPJQipJD+CS9GbW7ZqdgCsl6WHZkvSHpbshdsIl6c2sC+SZ\nijwgki4CdgXWkTSPbGF3l6Q3M0sKS8ARcUgfL7kkvZkZzR+CMLNefN9x93ICNiuZ7zvuXk7AZmYl\ncQI2MyuJE7CZWUmcgM3MSuIEbGZWEidgM7OSOAGbmZXECdjMrCROwGZmJXECNjMriROwWRfxuhOt\nxQnYrIt43YnW4gRsZoXw1XZjLZWAJe0l6SFJcySd0PgdZtaqfLXdWMskYEnLA2cBewNbAIdI2qLc\nqMysXbTjFXfLJGBgB2BORDwaEa8DFwPjS47JzNpEO15xF1aSaADWB56s2p8H7Ni7k6SJwMS0+5Kk\nh3Iefx3gr4OKsAFJQ3m4hvEO8efV1eCzhvTcNuF7LRVvC53Hvgz4/Jb03drp71quWAfweRvn6dRK\nCbjWN4xlGiImAZP6fXBpZkT0DCSwMrRTvO0UKzjeorVTvGXH2kpDEPOADav2NwCeKikWM7PCtVIC\nvh0YJ2mspBWBg4FpJcdkZlaYlhmCiIjFko4C/ggsD5wXEfcN4Uf0e9iiZO0UbzvFCo63aO0Ub6mx\nKmKZYVYzM2uCVhqCMDPrKk7AZmYl6agE3Ggqs6SVJF2SXr9V0pjmR7lUPI3i/YCkOyUtlnRAGTH2\niqdRvF+UdL+kuyVNl5TrXsii5Ij33yTdI2m2pBvLnnmZdyq+pAMkhaTSbp/KcW4/JWlBOrezJX2m\njDir4ml4biV9PP35vU/Sr5sSWER0xIPsh7tHgE2AFYG7gC169fkC8LO0fTBwSYvHOwbYEjgfOKAN\nzu9uwNvS9ufb4PyuXrW9L/CHVo439RsO3ADMAHpaNVbgU8BPyjqfA4h3HDALWDPtr9uM2DrpCjjP\nVObxwOS0PRXYQ82cKrS0hvFGxNyIuBt4s4wAe8kT73UR8UranUF2L3dZ8sS7qGp3VWpM/GmivFPx\nvwV8H3i1mcH10m7LBuSJ97PAWRHxAkBEPNuMwDopAdeayrx+X30iYjGwEFi7KdEtK0+8raS/8U4A\nfl9oRPXlilfSkZIeIUtqxzQptloaxitpG2DDiLiimYHVkPfPwr+k4aipkjas8Xqz5Il3c2BzSTdJ\nmiFpr2YE1kkJOM9U5lzTnZuklWLJI3e8kj4J9ACnFhpRfXmntp8VEZsCxwNfKzyqvtWNV9JywBnA\nl5oWUd/ynNvLgTERsSVwDUv+5VmGPPEOIxuG2BU4BDhH0oiC4+qoBJxnKvNbfSQNA9YAnm9KdMtq\nt6nXueKV9EHgJGDfiHitSbHV0t/zezGwX6ER1dco3uHAPwHXS5oL7ARMK+mHuIbnNiKeq/rv/z/A\ndk2KrZa8ueF3EfGPiHgMeIgsIRer7AHyIRxoHwY8CoxlyUD7u3v1OZKlf4Sb0srxVvX9JeX/CJfn\n/G5D9mPHuDb58zCuavujwMxWjrdX/+sp70e4POd2dNX2/sCMVj63wF7A5LS9DtmQxdqFx1bWSSno\nRO8D/F9KAieltm+SXY0BrAxcCswBbgM2afF4tyf7P/PLwHPAfS0e7zXAfGB2ekxr8Xh/BNyXYr2u\nXsJrhXh79S0tAec8t6ekc3tXOrfvbOVzSzZM8QPgfuAe4OBmxOWpyGZmJemkMWAzs7biBGxmVhIn\nYDOzkjgBm5mVxAnYzKwkLVMRw6wvkt4guzWoYr+ImFtSOGZDxrehWcuT9FJErFbn9WGRre1h1lY8\nBGFtKa03e6mky4GrUttXJN2eFoD5RlXfk9JasNdIukjSl1P79ZWpvJLWSVN8kbS8pFOrjvW51L5r\nes9USQ9KurCymp6k7SXdLOkuSbdJGi7pz5K2rorjJklbNuscWevzEIS1g1UkzU7bj0XE/mn7vcCW\nEfG8pD3J5u7vQDaraZqkD5DNIjyYbJr0MOBO4I4GnzcBWBgR20taCbhJ0lXptW2Ad5OtJXATsLOk\n24BLgIMi4nZJqwN/B84hWxf3OEmbAytFtryoGeAEbO3h7xGxdY32qyOispjSnukxK+2vRpaQhwOX\nRVqnWNK0HJ+3J7BlVRWSNdKxXgdui4h56VizyRbNXwg8HRG3w5J1hiVdCvynpK8AnyZb08PsLU7A\n1s5ertoWcEpE/Ly6g6Tj6HuZz8UsGYZbudexjo6IP/Y61q5A9Qpvb5D9HVKtz4iIVyRdTbb498fJ\nlug0e4vHgK1T/BH4tKTVACStL2ldsvI9+0taRdJwslXPKuayZJnEA3od6/OSVkjH2lzSqnU++0Hg\n7ZK2T/2Hp+VOIRuGOBO4vepq3QzwFbB1iIi4StK7gFvS72IvAZ+MiDslXUK24tnjwJ+r3nYaMEXS\nocC1Ve3nkA0t3Jl+ZFtAnbWCI+J1SQcBP5a0Ctn47weBlyLiDkmLgF8M0Ve1DuLb0KyrSPo6WWI8\nrUmf93aypSPfGRGtUNvPWoiHIMwKIukw4Fay9WedfG0ZvgI2MyuJr4DNzEriBGxmVhInYDOzkjgB\nm5mVxAnYzKwk/w+NsIEQ/2msowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "controversial_words = df[df.percentage < 0.10]\n",
    "plt.figure(figsize=(5, 3))\n",
    "plt.hist(controversial_words['df'], rwidth=0.8, edgecolor='black', facecolor=\"black\")\n",
    "plt.xlabel(\"Frequency\")\n",
    "plt.ylabel(\"Number of Stop Words\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"figures/google_document_frequency.png\", dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>freq</th>\n",
       "      <th>percentage</th>\n",
       "      <th>match_count</th>\n",
       "      <th>volume_count</th>\n",
       "      <th>df</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>book</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>122339388.0</td>\n",
       "      <td>7471834.0</td>\n",
       "      <td>0.575511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>day</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>235502808.0</td>\n",
       "      <td>7149547.0</td>\n",
       "      <td>0.550688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>early</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>131494753.0</td>\n",
       "      <td>6962309.0</td>\n",
       "      <td>0.536266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>570</th>\n",
       "      <td>finally</td>\n",
       "      <td>2</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>62381205.0</td>\n",
       "      <td>6909353.0</td>\n",
       "      <td>0.532187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>608</th>\n",
       "      <td>free</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>103078911.0</td>\n",
       "      <td>6511494.0</td>\n",
       "      <td>0.501542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>633</th>\n",
       "      <td>general</td>\n",
       "      <td>2</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>225306151.0</td>\n",
       "      <td>8088215.0</td>\n",
       "      <td>0.622988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>great</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>278574742.0</td>\n",
       "      <td>8087649.0</td>\n",
       "      <td>0.622944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>762</th>\n",
       "      <td>house</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>158321573.0</td>\n",
       "      <td>6886583.0</td>\n",
       "      <td>0.530433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>introduction</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>37676373.0</td>\n",
       "      <td>6642548.0</td>\n",
       "      <td>0.511636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>936</th>\n",
       "      <td>life</td>\n",
       "      <td>2</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>303632580.0</td>\n",
       "      <td>7894850.0</td>\n",
       "      <td>0.608094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>985</th>\n",
       "      <td>man</td>\n",
       "      <td>4</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>304508612.0</td>\n",
       "      <td>7027563.0</td>\n",
       "      <td>0.541292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1136</th>\n",
       "      <td>note</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>72585815.0</td>\n",
       "      <td>6606005.0</td>\n",
       "      <td>0.508822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>number</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>210832441.0</td>\n",
       "      <td>6581539.0</td>\n",
       "      <td>0.506937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262</th>\n",
       "      <td>people</td>\n",
       "      <td>2</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>329358030.0</td>\n",
       "      <td>7311073.0</td>\n",
       "      <td>0.563129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1533</th>\n",
       "      <td>small</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>175242603.0</td>\n",
       "      <td>6641887.0</td>\n",
       "      <td>0.511585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1566</th>\n",
       "      <td>special</td>\n",
       "      <td>4</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>86374118.0</td>\n",
       "      <td>6531715.0</td>\n",
       "      <td>0.503100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1580</th>\n",
       "      <td>state</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>276687623.0</td>\n",
       "      <td>7876471.0</td>\n",
       "      <td>0.606678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1582</th>\n",
       "      <td>states</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>189702937.0</td>\n",
       "      <td>7219768.0</td>\n",
       "      <td>0.556096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1754</th>\n",
       "      <td>time</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>583894803.0</td>\n",
       "      <td>8118768.0</td>\n",
       "      <td>0.625341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1755</th>\n",
       "      <td>times</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>122137394.0</td>\n",
       "      <td>6542576.0</td>\n",
       "      <td>0.503936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>work</td>\n",
       "      <td>4</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>310897825.0</td>\n",
       "      <td>7098787.0</td>\n",
       "      <td>0.546778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012</th>\n",
       "      <td>world</td>\n",
       "      <td>5</td>\n",
       "      <td>0.096154</td>\n",
       "      <td>252934367.0</td>\n",
       "      <td>7676228.0</td>\n",
       "      <td>0.591255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2036</th>\n",
       "      <td>years</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>281623235.0</td>\n",
       "      <td>6562310.0</td>\n",
       "      <td>0.505456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              word  freq  percentage  match_count  volume_count        df\n",
       "238           book     3    0.057692  122339388.0     7471834.0  0.575511\n",
       "368            day     5    0.096154  235502808.0     7149547.0  0.550688\n",
       "455          early     5    0.096154  131494753.0     6962309.0  0.536266\n",
       "570        finally     2    0.038462   62381205.0     6909353.0  0.532187\n",
       "608           free     1    0.019231  103078911.0     6511494.0  0.501542\n",
       "633        general     2    0.038462  225306151.0     8088215.0  0.622988\n",
       "657          great     3    0.057692  278574742.0     8087649.0  0.622944\n",
       "762          house     1    0.019231  158321573.0     6886583.0  0.530433\n",
       "847   introduction     1    0.019231   37676373.0     6642548.0  0.511636\n",
       "936           life     2    0.038462  303632580.0     7894850.0  0.608094\n",
       "985            man     4    0.076923  304508612.0     7027563.0  0.541292\n",
       "1136          note     1    0.019231   72585815.0     6606005.0  0.508822\n",
       "1156        number     5    0.096154  210832441.0     6581539.0  0.506937\n",
       "1262        people     2    0.038462  329358030.0     7311073.0  0.563129\n",
       "1533         small     5    0.096154  175242603.0     6641887.0  0.511585\n",
       "1566       special     4    0.076923   86374118.0     6531715.0  0.503100\n",
       "1580         state     3    0.057692  276687623.0     7876471.0  0.606678\n",
       "1582        states     3    0.057692  189702937.0     7219768.0  0.556096\n",
       "1754          time     5    0.096154  583894803.0     8118768.0  0.625341\n",
       "1755         times     1    0.019231  122137394.0     6542576.0  0.503936\n",
       "2007          work     4    0.076923  310897825.0     7098787.0  0.546778\n",
       "2012         world     5    0.096154  252934367.0     7676228.0  0.591255\n",
       "2036         years     3    0.057692  281623235.0     6562310.0  0.505456"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "controversial_words[controversial_words.df > 0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
